{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8.0\n",
      "1.14.0\n",
      "0.22.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# Import libraries and modules\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shutil\n",
    "print tf.__version__\n",
    "print np.__version__\n",
    "print pd.__version__\n",
    "np.set_printoptions(threshold=np.inf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# change these to try this notebook out\n",
    "BUCKET = 'youtube8m-4-train'\n",
    "PROJECT = 'qwiklabs-gcp-8d3d0cd07cef9252'\n",
    "REGION = 'us-central1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Import os environment variables\n",
    "import os\n",
    "os.environ['BUCKET'] = BUCKET\n",
    "os.environ['PROJECT'] = PROJECT\n",
    "os.environ['REGION'] = REGION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Local Development"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Set logging to be level of INFO\n",
    "tf.logging.set_verbosity(tf.logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# The number of video classes\n",
    "NUM_CLASSES = 4716"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "arguments = {}\n",
    "arguments[\"train_file_pattern\"] = \"gs://youtube-8m-team/1/video_level/train/train*.tfrecord\"\n",
    "arguments[\"eval_file_pattern\"] = \"gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\"\n",
    "arguments[\"output_dir\"] = \"trained_model\"\n",
    "arguments[\"batch_size\"] = 10\n",
    "arguments[\"train_steps\"] = 100\n",
    "arguments[\"hidden_units\"] = [1024, 256, 64]\n",
    "arguments[\"top_k\"] = 5\n",
    "arguments[\"start_delay_secs\"] = 60\n",
    "arguments[\"throttle_secs\"] = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create an input function to read our training and validation data\n",
    "# Then provide the results to the Estimator API\n",
    "def read_dataset_video(file_pattern, mode, batch_size):\n",
    "  def _input_fn():\n",
    "    print(\"\\nread_dataset_video: _input_fn: file_pattern = {}\".format(file_pattern))\n",
    "    print(\"read_dataset_video: _input_fn: mode = {}\".format(mode))\n",
    "    print(\"read_dataset_video: _input_fn: batch_size = {}\".format(batch_size))\n",
    "\n",
    "    # Read files from file_pattern which provided by args\n",
    "    input_file_names = tf.matching_files(file_pattern)\n",
    "\n",
    "    # Determine amount of times to repeat file and if we should shuffle the file queue based on if we are training or evaluating\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "      num_epochs = None # forever\n",
    "      shuffle = True\n",
    "    else:\n",
    "      num_epochs = 1 # until EOF\n",
    "      shuffle = False\n",
    "\n",
    "    # Create filename queue from our input file names\n",
    "    filename_queue = tf.train.string_input_producer(string_tensor = input_file_names, num_epochs = num_epochs, shuffle = shuffle)\n",
    "    \n",
    "    # Create a TF Record reader to read in our TF Record files\n",
    "    reader = tf.TFRecordReader()\n",
    "    \n",
    "    # Use our TF Record reader to read from the filename queue\n",
    "    queue, serialized_examples = reader.read(queue = filename_queue)\n",
    "    \n",
    "    # Create feature map\n",
    "    feature_map = {\n",
    "      'video_id': tf.FixedLenFeature(shape = [], dtype = tf.string),\n",
    "      'labels': tf.VarLenFeature(dtype = tf.int64),\n",
    "      'mean_rgb': tf.FixedLenFeature(shape = [1024], dtype = tf.float32),\n",
    "      'mean_audio': tf.FixedLenFeature(shape = [128], dtype = tf.float32)\n",
    "    }\n",
    "\n",
    "    # Parse TF Records into our features\n",
    "    features = tf.parse_single_example(serialized = serialized_examples, features = feature_map)\n",
    "    print(\"read_dataset_video: _input_fn: features = {}\".format(features)) # shape = video_id = (), mean_rgb = (1024,), mean_audio = (128,), labels = SparseTensor object\n",
    "    \n",
    "    # Add labels to features dictionary and change to correct format from sparse to dense and to floats\n",
    "    features['labels'] = tf.cast(x = tf.sparse_to_dense(sparse_indices = features['labels'].values, output_shape = (NUM_CLASSES,), sparse_values = 1, validate_indices = False), dtype = tf.float32)\n",
    "    print(\"read_dataset_video: _input_fn: features = {}\".format(features)) # shape = video_id = (), mean_rgb = (1024,), mean_audio = (128,), labels = (NUM_CLASSES,)\n",
    "\n",
    "    # Shuffle and batch features\n",
    "    batch_features = tf.train.shuffle_batch(\n",
    "      tensors = features, \n",
    "      batch_size = batch_size, \n",
    "      capacity = batch_size * 10, \n",
    "      min_after_dequeue = batch_size,\n",
    "      num_threads = 1,\n",
    "      enqueue_many = False,\n",
    "      allow_smaller_final_batch = True)\n",
    "    print(\"read_dataset_video: _input_fn: batch_features = {}\".format(batch_features)) # shape = video_id = (batch_size,), mean_rgb = (batch_size, 1024), mean_audio = (batch_size, 128), labels = (batch_size, NUM_CLASSES)\n",
    "\n",
    "    # Pop off labels from feature dictionary\n",
    "    batch_labels = batch_features.pop('labels')\n",
    "    print(\"read_dataset_video: _input_fn: batch_labels = {}\".format(batch_labels)) # shape = (batch_size, NUM_CLASSES)\n",
    "\n",
    "    return batch_features, batch_labels\n",
    "  return _input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our model function to be used in our custom estimator\n",
    "def video_level_model(features, labels, mode, params):\n",
    "  print(\"\\nvideo_level_model: features = {}\".format(features))\n",
    "  print(\"video_level_model: labels = {}\".format(labels))\n",
    "  print(\"video_level_model: mode = {}\".format(mode))\n",
    "\n",
    "  # 0. Configure network\n",
    "  # Get dynamic batch size\n",
    "  current_batch_size = tf.shape(features['mean_rgb'])[0]\n",
    "  print(\"video_level_model: current_batch_size = {}\".format(current_batch_size))\n",
    "\n",
    "  # Stack all of the features into a 3-D tensor\n",
    "  combined_features = tf.concat(values = [features['mean_rgb'], features['mean_audio']], axis = 1) # shape = (current_batch_size, 1024 + 128)\n",
    "  print(\"video_level_model: combined_features = {}\".format(combined_features))\n",
    "\n",
    "  # 1. Create the DNN structure now\n",
    "  # Create the input layer to our frame DNN\n",
    "  network = combined_features # shape = (current_batch_size, 1024 + 128)\n",
    "  print(\"video_level_model: network = combined_features = {}\".format(network))\n",
    "\n",
    "  # Add hidden layers with the given number of units/neurons per layer\n",
    "  for units in params['hidden_units']:\n",
    "    network = tf.layers.dense(inputs = network, units = units, activation = tf.nn.relu) # shape = (current_batch_size, units)\n",
    "    print(\"video_level_model: network = {}, units = {}\".format(network, units))\n",
    "\n",
    "  # Connect the final hidden layer to a dense layer with no activation to get the logits\n",
    "  logits = tf.layers.dense(inputs = network, units = NUM_CLASSES, activation = None) # shape = (current_batch_size, NUM_CLASSES)\n",
    "  print(\"video_level_model: logits = {}\".format(logits))\n",
    "\n",
    "  # Select the top k logits in descending order\n",
    "  top_k_logits = tf.nn.top_k(input = logits, k = params['top_k'], sorted = True) # shape = (current_batch_size, top_k)\n",
    "  print(\"video_level_model: top_k_logits = {}\".format(top_k_logits))\n",
    "\n",
    "  # Since this is a multi-class, multi-label problem we will apply a sigmoid, not a softmax, to each logit to get its own probability\n",
    "  probabilities = tf.sigmoid(logits) # shape = (current_batch_size, NUM_CLASSES)\n",
    "  print(\"video_level_model: probabilities = {}\".format(probabilities))\n",
    "\n",
    "  # Select the top k probabilities in descending order\n",
    "  top_k_probabilities = tf.sigmoid(top_k_logits.values) # shape = (current_batch_size, top_k)\n",
    "  print(\"video_level_model: top_k_probabilities = {}\".format(top_k_probabilities))\n",
    "\n",
    "  # Select the top k classes in descending order of likelihood\n",
    "  top_k_classes = top_k_logits.indices # shape = (current_batch_size, top_k)\n",
    "  print(\"video_level_model: top_k_classes = {}\".format(top_k_classes))\n",
    "\n",
    "  # The 0/1 predictions based on a threshold, in this case the threshold is if the probability it greater than random chance\n",
    "  predictions = tf.where(\n",
    "    condition = probabilities > 1.0 / NUM_CLASSES, # shape = (current_batch_size, NUM_CLASSES)\n",
    "    x = tf.ones_like(tensor = probabilities), \n",
    "    y = tf.zeros_like(tensor = probabilities))\n",
    "  print(\"video_level_model: predictions = {}\".format(predictions))\n",
    "\n",
    "  top_k_predictions = tf.where(\n",
    "    condition = top_k_probabilities > 1.0 / NUM_CLASSES, # shape = (current_batch_size, top_k)\n",
    "    x = tf.ones_like(tensor = top_k_probabilities), \n",
    "    y = tf.zeros_like(tensor = top_k_probabilities))\n",
    "  print(\"video_level_model: top_k_predictions = {}\\n\".format(top_k_predictions))\n",
    "\n",
    "  # 2. Loss function, training/eval ops \n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    # Since this is a multi-class, multi-label problem, we will use sigmoid activation and cross entropy loss\n",
    "    loss = tf.losses.sigmoid_cross_entropy(multi_class_labels = labels, logits = logits)\n",
    "\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = 0.01,\n",
    "      optimizer = \"Adam\")\n",
    "    eval_metric_ops = {\n",
    "      \"accuracy\": tf.metrics.mean_per_class_accuracy(labels = labels, predictions = predictions, num_classes = NUM_CLASSES)\n",
    "    }\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "\n",
    "  # 3. Create predictions\n",
    "  predictions_dict = {\"logits\": top_k_logits.values, \n",
    "                      \"probabilities\": top_k_probabilities, \n",
    "                      \"predictions\": top_k_predictions,\n",
    "                      \"classes\": top_k_classes}\n",
    "\n",
    "  # 4. Create export outputs\n",
    "  export_outputs = {\"predict_export_outputs\": tf.estimator.export.PredictOutput(outputs = predictions_dict)}\n",
    "\n",
    "  # 5. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "      mode = mode,\n",
    "      predictions = predictions_dict,\n",
    "      loss = loss,\n",
    "      train_op = train_op,\n",
    "      eval_metric_ops = eval_metric_ops,\n",
    "      export_outputs = export_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create our serving input function to accept the data at serving and send it in the right format to our custom estimator\n",
    "def serving_input_fn():\n",
    "  # This function fixes the shape and type of our input strings\n",
    "  def fix_shape_and_type_for_serving(placeholder):\n",
    "    # String split each string in the batch and output the values from the resulting SparseTensors\n",
    "    split_string = tf.map_fn(\n",
    "      fn = lambda x: tf.string_split(source = [placeholder[x]], delimiter=',').values, \n",
    "      elems = tf.range(start = 0, limit = tf.shape(input = placeholder)[0]), \n",
    "      dtype = tf.string) # shape = (batch_size, input_sequence_length)\n",
    "    print(\"serving_input_fn: fix_shape_and_type_for_serving: split_string = {}\".format(split_string))\n",
    "\n",
    "    # Convert each string in the split tensor to float\n",
    "    feature_tensor = tf.string_to_number(string_tensor = split_string, out_type = tf.float32) # shape = (batch_size, input_sequence_length)\n",
    "    print(\"serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = {}\".format(feature_tensor))\n",
    "    return feature_tensor\n",
    "  \n",
    "  # This function fixes dynamic shape ambiguity of last dimension so that we will be able to use it in our DNN (since tf.layers.dense require the last dimension to be known)\n",
    "  def get_shape_and_set_modified_shape_2D(tensor, additional_dimension_sizes):\n",
    "    # Get static shape for tensor and convert it to list\n",
    "    shape = tensor.get_shape().as_list()\n",
    "    # Set outer shape to additional_dimension_sizes[0] since we know that this is the correct size\n",
    "    shape[1] = additional_dimension_sizes[0]\n",
    "    # Set the shape of tensor to our modified shape\n",
    "    tensor.set_shape(shape = shape) # shape = (batch_size, additional_dimension_sizes[0])\n",
    "    print(\"serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = {}, additional_dimension_sizes = {}\".format(tensor, additional_dimension_sizes))\n",
    "    return tensor\n",
    "  \n",
    "  # Create placeholders to accept the data sent to the model at serving time\n",
    "  feature_placeholders = { # all features come in as a batch of strings, shape = (batch_size,), this was so because of passing the arrays to online ml-engine prediction\n",
    "    'video_id': tf.placeholder(dtype = tf.string, shape = [None]),\n",
    "    'mean_rgb': tf.placeholder(dtype = tf.string, shape = [None]),\n",
    "    'mean_audio': tf.placeholder(dtype = tf.string, shape = [None])\n",
    "  }\n",
    "  print(\"\\nserving_input_fn: feature_placeholders = {}\".format(feature_placeholders))\n",
    "\n",
    "  # Create feature tensors\n",
    "  features = {\n",
    "    \"video_id\": feature_placeholders[\"video_id\"],\n",
    "    \"mean_rgb\": fix_shape_and_type_for_serving(placeholder = feature_placeholders[\"mean_rgb\"]),\n",
    "    \"mean_audio\": fix_shape_and_type_for_serving(placeholder = feature_placeholders[\"mean_audio\"])\n",
    "  }\n",
    "  print(\"serving_input_fn: features = {}\".format(features))\n",
    "\n",
    "  # Fix dynamic shape ambiguity of feature tensors for our DNN\n",
    "  features[\"mean_rgb\"] = get_shape_and_set_modified_shape_2D(tensor = features[\"mean_rgb\"], additional_dimension_sizes = [1024])\n",
    "  features[\"mean_audio\"] = get_shape_and_set_modified_shape_2D(tensor = features[\"mean_audio\"], additional_dimension_sizes = [128])\n",
    "  print(\"serving_input_fn: features = {}\\n\".format(features))\n",
    "  \n",
    "  return tf.estimator.export.ServingInputReceiver(features = features, receiver_tensors = feature_placeholders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create custom estimator's train and evaluate function\n",
    "def train_and_evaluate(args):\n",
    "  # Create custom estimator's train and evaluate function\n",
    "  estimator = tf.estimator.Estimator(\n",
    "    model_fn = video_level_model, \n",
    "    model_dir = args['output_dir'],\n",
    "    params = {'hidden_units': args['hidden_units'], 'top_k': args['top_k']})\n",
    "  # Create train spec to read in our training data\n",
    "  train_spec = tf.estimator.TrainSpec(\n",
    "    input_fn = read_dataset_video(\n",
    "      file_pattern = args['train_file_pattern'], \n",
    "      mode = tf.estimator.ModeKeys.TRAIN, \n",
    "      batch_size = args['batch_size']),\n",
    "    max_steps = args['train_steps'])\n",
    "  # Create exporter to save out the complete model to disk\n",
    "  exporter = tf.estimator.LatestExporter(name = 'exporter', serving_input_receiver_fn = serving_input_fn)\n",
    "  # Create eval spec to read in our validation data and export our model\n",
    "  eval_spec = tf.estimator.EvalSpec(\n",
    "    input_fn = read_dataset_video(\n",
    "      file_pattern = args['eval_file_pattern'], \n",
    "      mode = tf.estimator.ModeKeys.EVAL, \n",
    "      batch_size = args['batch_size']),\n",
    "    steps = None,\n",
    "    exporters = exporter,\n",
    "    start_delay_secs = args['start_delay_secs'],\n",
    "    throttle_secs = args['throttle_secs'])\n",
    "  # Create train and evaluate loop to train and evaluate our estimator\n",
    "  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_train_distribute': None, '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f0d50b2a250>, '_evaluation_master': '', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': 'trained_model', '_global_id_in_cluster': 0, '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after 30 secs (eval_spec.throttle_secs) or training is finished.\n",
      "\n",
      "read_dataset_video: _input_fn: file_pattern = gs://youtube-8m-team/1/video_level/train/train*.tfrecord\n",
      "read_dataset_video: _input_fn: mode = train\n",
      "read_dataset_video: _input_fn: batch_size = 10\n",
      "read_dataset_video: _input_fn: features = {'labels': <tensorflow.python.framework.sparse_tensor.SparseTensor object at 0x7f0d50b2a1d0>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: features = {'labels': <tf.Tensor 'Cast:0' shape=(4716,) dtype=float32>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'labels': <tf.Tensor 'shuffle_batch:0' shape=(?, 4716) dtype=float32>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "video_level_model: features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "video_level_model: labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "video_level_model: mode = train\n",
      "video_level_model: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "video_level_model: combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "video_level_model: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "video_level_model: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "video_level_model: logits = Tensor(\"dense_3/BiasAdd:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_logits = TopKV2(values=<tf.Tensor 'TopKV2:0' shape=(?, 5) dtype=float32>, indices=<tf.Tensor 'TopKV2:1' shape=(?, 5) dtype=int32>)\n",
      "video_level_model: probabilities = Tensor(\"Sigmoid:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_probabilities = Tensor(\"Sigmoid_1:0\", shape=(?, 5), dtype=float32)\n",
      "video_level_model: top_k_classes = Tensor(\"TopKV2:1\", shape=(?, 5), dtype=int32)\n",
      "video_level_model: predictions = Tensor(\"Select:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_predictions = Tensor(\"Select_1:0\", shape=(?, 5), dtype=float32)\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.6935538, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 100 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0066434606.\n",
      "\n",
      "read_dataset_video: _input_fn: file_pattern = gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\n",
      "read_dataset_video: _input_fn: mode = eval\n",
      "read_dataset_video: _input_fn: batch_size = 10\n",
      "read_dataset_video: _input_fn: features = {'labels': <tensorflow.python.framework.sparse_tensor.SparseTensor object at 0x7f0d50697c50>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: features = {'labels': <tf.Tensor 'Cast:0' shape=(4716,) dtype=float32>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'labels': <tf.Tensor 'shuffle_batch:0' shape=(?, 4716) dtype=float32>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "video_level_model: features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "video_level_model: labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "video_level_model: mode = eval\n",
      "video_level_model: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "video_level_model: combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "video_level_model: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "video_level_model: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "video_level_model: logits = Tensor(\"dense_3/BiasAdd:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_logits = TopKV2(values=<tf.Tensor 'TopKV2:0' shape=(?, 5) dtype=float32>, indices=<tf.Tensor 'TopKV2:1' shape=(?, 5) dtype=int32>)\n",
      "video_level_model: probabilities = Tensor(\"Sigmoid:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_probabilities = Tensor(\"Sigmoid_1:0\", shape=(?, 5), dtype=float32)\n",
      "video_level_model: top_k_classes = Tensor(\"TopKV2:1\", shape=(?, 5), dtype=int32)\n",
      "video_level_model: predictions = Tensor(\"Select:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_predictions = Tensor(\"Select_1:0\", shape=(?, 5), dtype=float32)\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-25-05:06:52\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-100\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-25-05:06:54\n",
      "INFO:tensorflow:Saving dict for global step 100: accuracy = 0.0003188998, global_step = 100, loss = 0.008378749\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'Placeholder_2:0' shape=(?,) dtype=string>, 'mean_rgb': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber_1:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: features = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'StringToNumber_1:0' shape=(?, ?) dtype=float32>, 'mean_rgb': <tf.Tensor 'StringToNumber:0' shape=(?, ?) dtype=float32>}\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber:0\", shape=(?, 1024), dtype=float32), additional_dimension_sizes = [1024]\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber_1:0\", shape=(?, 128), dtype=float32), additional_dimension_sizes = [128]\n",
      "serving_input_fn: features = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'StringToNumber_1:0' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'StringToNumber:0' shape=(?, 1024) dtype=float32>}\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "\n",
      "video_level_model: features = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'StringToNumber_1:0' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'StringToNumber:0' shape=(?, 1024) dtype=float32>}\n",
      "video_level_model: labels = None\n",
      "video_level_model: mode = infer\n",
      "video_level_model: current_batch_size = Tensor(\"strided_slice_2:0\", shape=(), dtype=int32)\n",
      "video_level_model: combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "video_level_model: network = Tensor(\"dense_1/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "video_level_model: network = Tensor(\"dense_2/Relu:0\", shape=(?, 64), dtype=float32), units = 64\n",
      "video_level_model: logits = Tensor(\"dense_3/BiasAdd:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_logits = TopKV2(values=<tf.Tensor 'TopKV2:0' shape=(?, 5) dtype=float32>, indices=<tf.Tensor 'TopKV2:1' shape=(?, 5) dtype=int32>)\n",
      "video_level_model: probabilities = Tensor(\"Sigmoid:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_probabilities = Tensor(\"Sigmoid_1:0\", shape=(?, 5), dtype=float32)\n",
      "video_level_model: top_k_classes = Tensor(\"TopKV2:1\", shape=(?, 5), dtype=int32)\n",
      "video_level_model: predictions = Tensor(\"Select:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_predictions = Tensor(\"Select_1:0\", shape=(?, 5), dtype=float32)\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default', 'predict_export_outputs']\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-100\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-1527224814/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "# Run the training job\n",
    "shutil.rmtree(arguments['output_dir'], ignore_errors = True) # start fresh each time\n",
    "train_and_evaluate(args = arguments)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "read_dataset_video: _input_fn: file_pattern = gs://youtube-8m-team/1/video_level/train/train*.tfrecord\n",
      "read_dataset_video: _input_fn: mode = train\n",
      "read_dataset_video: _input_fn: batch_size = 10\n",
      "read_dataset_video: _input_fn: features = {'labels': <tensorflow.python.framework.sparse_tensor.SparseTensor object at 0x7fabc6e679d0>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: features = {'labels': <tf.Tensor 'Cast:0' shape=(4716,) dtype=float32>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'labels': <tf.Tensor 'shuffle_batch:0' shape=(?, 4716) dtype=float32>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "\n",
      "video_level_model: features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "video_level_model: labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "video_level_model: mode = train\n",
      "video_level_model: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "video_level_model: combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "video_level_model: network = Tensor(\"dense_1/Relu:0\", shape=(?, 512), dtype=float32), units = 512\n",
      "video_level_model: network = Tensor(\"dense_2/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "video_level_model: logits = Tensor(\"dense_3/BiasAdd:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_logits = TopKV2(values=<tf.Tensor 'TopKV2:0' shape=(?, 5) dtype=float32>, indices=<tf.Tensor 'TopKV2:1' shape=(?, 5) dtype=int32>)\n",
      "video_level_model: probabilities = Tensor(\"Sigmoid:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_probabilities = Tensor(\"Sigmoid_1:0\", shape=(?, 5), dtype=float32)\n",
      "video_level_model: top_k_classes = Tensor(\"TopKV2:1\", shape=(?, 5), dtype=int32)\n",
      "video_level_model: predictions = Tensor(\"Select:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_predictions = Tensor(\"Select_1:0\", shape=(?, 5), dtype=float32)\n",
      "\n",
      "\n",
      "read_dataset_video: _input_fn: file_pattern = gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\n",
      "read_dataset_video: _input_fn: mode = eval\n",
      "read_dataset_video: _input_fn: batch_size = 10\n",
      "read_dataset_video: _input_fn: features = {'labels': <tensorflow.python.framework.sparse_tensor.SparseTensor object at 0x7fabbd051850>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: features = {'labels': <tf.Tensor 'Cast:0' shape=(4716,) dtype=float32>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'labels': <tf.Tensor 'shuffle_batch:0' shape=(?, 4716) dtype=float32>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "\n",
      "video_level_model: features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "video_level_model: labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32, device=/device:CPU:0)\n",
      "video_level_model: mode = eval\n",
      "video_level_model: current_batch_size = Tensor(\"strided_slice:0\", shape=(), dtype=int32)\n",
      "video_level_model: combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "video_level_model: network = Tensor(\"dense_1/Relu:0\", shape=(?, 512), dtype=float32), units = 512\n",
      "video_level_model: network = Tensor(\"dense_2/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "video_level_model: logits = Tensor(\"dense_3/BiasAdd:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_logits = TopKV2(values=<tf.Tensor 'TopKV2:0' shape=(?, 5) dtype=float32>, indices=<tf.Tensor 'TopKV2:1' shape=(?, 5) dtype=int32>)\n",
      "video_level_model: probabilities = Tensor(\"Sigmoid:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_probabilities = Tensor(\"Sigmoid_1:0\", shape=(?, 5), dtype=float32)\n",
      "video_level_model: top_k_classes = Tensor(\"TopKV2:1\", shape=(?, 5), dtype=int32)\n",
      "video_level_model: predictions = Tensor(\"Select:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_predictions = Tensor(\"Select_1:0\", shape=(?, 5), dtype=float32)\n",
      "\n",
      "\n",
      "serving_input_fn: feature_placeholders = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'Placeholder_2:0' shape=(?,) dtype=string>, 'mean_rgb': <tf.Tensor 'Placeholder_1:0' shape=(?,) dtype=string>}\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: split_string = Tensor(\"map_1/TensorArrayStack/TensorArrayGatherV3:0\", shape=(?, ?), dtype=string)\n",
      "serving_input_fn: fix_shape_and_type_for_serving: feature_tensor = Tensor(\"StringToNumber_1:0\", shape=(?, ?), dtype=float32)\n",
      "serving_input_fn: features = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'StringToNumber_1:0' shape=(?, ?) dtype=float32>, 'mean_rgb': <tf.Tensor 'StringToNumber:0' shape=(?, ?) dtype=float32>}\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber:0\", shape=(?, 1024), dtype=float32), additional_dimension_sizes = [1024]\n",
      "serving_input_fn: get_shape_and_set_modified_shape_2D: tensor = Tensor(\"StringToNumber_1:0\", shape=(?, 128), dtype=float32), additional_dimension_sizes = [128]\n",
      "serving_input_fn: features = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'StringToNumber_1:0' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'StringToNumber:0' shape=(?, 1024) dtype=float32>}\n",
      "\n",
      "\n",
      "video_level_model: features = {'video_id': <tf.Tensor 'Placeholder:0' shape=(?,) dtype=string>, 'mean_audio': <tf.Tensor 'StringToNumber_1:0' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'StringToNumber:0' shape=(?, 1024) dtype=float32>}\n",
      "video_level_model: labels = None\n",
      "video_level_model: mode = infer\n",
      "video_level_model: current_batch_size = Tensor(\"strided_slice_2:0\", shape=(), dtype=int32)\n",
      "video_level_model: combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = combined_features = Tensor(\"concat:0\", shape=(?, 1152), dtype=float32)\n",
      "video_level_model: network = Tensor(\"dense/Relu:0\", shape=(?, 1024), dtype=float32), units = 1024\n",
      "video_level_model: network = Tensor(\"dense_1/Relu:0\", shape=(?, 512), dtype=float32), units = 512\n",
      "video_level_model: network = Tensor(\"dense_2/Relu:0\", shape=(?, 256), dtype=float32), units = 256\n",
      "video_level_model: logits = Tensor(\"dense_3/BiasAdd:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_logits = TopKV2(values=<tf.Tensor 'TopKV2:0' shape=(?, 5) dtype=float32>, indices=<tf.Tensor 'TopKV2:1' shape=(?, 5) dtype=int32>)\n",
      "video_level_model: probabilities = Tensor(\"Sigmoid:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_probabilities = Tensor(\"Sigmoid_1:0\", shape=(?, 5), dtype=float32)\n",
      "video_level_model: top_k_classes = Tensor(\"TopKV2:1\", shape=(?, 5), dtype=int32)\n",
      "video_level_model: predictions = Tensor(\"Select:0\", shape=(?, 4716), dtype=float32)\n",
      "video_level_model: top_k_predictions = Tensor(\"Select_1:0\", shape=(?, 5), dtype=float32)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_task_type': 'worker', '_train_distribute': None, '_is_chief': True, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fabc6f05f10>, '_evaluation_master': '', '_save_checkpoints_steps': None, '_keep_checkpoint_every_n_hours': 10000, '_service': None, '_num_ps_replicas': 0, '_tf_random_seed': None, '_master': '', '_num_worker_replicas': 1, '_task_id': 0, '_log_step_count_steps': 100, '_model_dir': 'trained_model/', '_global_id_in_cluster': 0, '_save_summary_steps': 100}\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after 30 secs (eval_spec.throttle_secs) or training is finished.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "2018-05-25 05:07:31.666780: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.6938704, step = 1\n",
      "INFO:tensorflow:Saving checkpoints for 100 into trained_model/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.0080804285.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-05-25-05:07:45\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-100\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-05-25-05:07:48\n",
      "INFO:tensorflow:Saving dict for global step 100: accuracy = 0.00030351995, global_step = 100, loss = 0.0103987595\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default', 'predict_export_outputs']\n",
      "INFO:tensorflow:Restoring parameters from trained_model/model.ckpt-100\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: trained_model/export/exporter/temp-1527224870/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "OUTDIR=trained_model\n",
    "rm -rf $OUTDIR\n",
    "export PYTHONPATH=$PYTHONPATH:$PWD/trainer\n",
    "python -m trainer.task \\\n",
    "  --train_file_pattern=\"gs://youtube-8m-team/1/video_level/train/train*.tfrecord\" \\\n",
    "  --eval_file_pattern=\"gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\"  \\\n",
    "  --output_dir=$OUTDIR \\\n",
    "  --batch_size=10 \\\n",
    "  --train_steps=100 \\\n",
    "  --hidden_units=\"1024 512 256\" \\\n",
    "  --top_k=5 \\\n",
    "  --job-dir=./tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://youtube8m-4-train/youtube_8m_video_level/trained_model us-central1 job_youtube_8m_video_level180525_010112\n",
      "jobId: job_youtube_8m_video_level180525_010112\n",
      "state: QUEUED\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "CommandException: 1 files/objects could not be removed.\n",
      "Job [job_youtube_8m_video_level180525_010112] submitted successfully.\n",
      "Your job is still active. You may view the status of your job with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs describe job_youtube_8m_video_level180525_010112\n",
      "\n",
      "or continue streaming the logs with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs stream-logs job_youtube_8m_video_level180525_010112\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "OUTDIR=gs://$BUCKET/youtube_8m_video_level/trained_model\n",
    "JOBNAME=job_youtube_8m_video_level$(date -u +%y%m%d_%H%M%S)\n",
    "echo $OUTDIR $REGION $JOBNAME\n",
    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",    "gcloud ml-engine jobs submit training $JOBNAME \\\n",
    "  --region=$REGION \\\n",
    "  --module-name=trainer.task \\\n",
    "  --package-path=$PWD/trainer \\\n",
    "  --job-dir=$OUTDIR \\\n",
    "  --staging-bucket=gs://$BUCKET \\\n",
    "  --scale-tier=STANDARD_1 \\\n",
    "  --runtime-version=1.5 \\\n",
    "  -- \\\n",
    "  --train_file_pattern=\"gs://youtube-8m-team/1/video_level/train/train*.tfrecord\" \\\n",
    "  --eval_file_pattern=\"gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\"  \\\n",
    "  --output_dir=$OUTDIR \\\n",
    "  --batch_size=50 \\\n",
    "  --train_steps=10000 \\\n",
    "  --hidden_units=\"1024 512 256\" \\\n",
    "  --top_k=5 \\\n",
    "  --job-dir=$OUTDIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing hyperparam.yaml\n"
     ]
    }
   ],
   "source": [
    "%writefile hyperparam.yaml\n",
    "trainingInput:\n",
    "  scaleTier: STANDARD_1\n",
    "  hyperparameters:\n",
    "    hyperparameterMetricTag: accuracy\n",
    "    goal: MAXIMIZE\n",
    "    maxTrials: 30\n",
    "    maxParallelTrials: 1\n",
    "    params:\n",
    "    - parameterName: batch_size\n",
    "      type: INTEGER\n",
    "      minValue: 8\n",
    "      maxValue: 512\n",
    "      scaleType: UNIT_LOG_SCALE\n",
    "    - parameterName: hidden_units\n",
    "      type: CATEGORICAL\n",
    "      categoricalValues: [\"64 32\", \"256 128 16\", \"64 64 64 8\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gs://youtube8m-4-train/youtube_8m_video_level/hyperparam us-central1 job_youtube_8m_video_level180525_061702\n",
      "jobId: job_youtube_8m_video_level180525_061702\n",
      "state: QUEUED\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "CommandException: 1 files/objects could not be removed.\n",
      "Job [job_youtube_8m_video_level180525_061702] submitted successfully.\n",
      "Your job is still active. You may view the status of your job with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs describe job_youtube_8m_video_level180525_061702\n",
      "\n",
      "or continue streaming the logs with the command\n",
      "\n",
      "  $ gcloud ml-engine jobs stream-logs job_youtube_8m_video_level180525_061702\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "OUTDIR=gs://$BUCKET/youtube_8m_video_level/hyperparam\n",
    "JOBNAME=job_youtube_8m_video_level$(date -u +%y%m%d_%H%M%S)\n",
    "echo $OUTDIR $REGION $JOBNAME\n",
    "gcloud storage rm --recursive --continue-on-error $OUTDIR\n",    "gcloud ml-engine jobs submit training $JOBNAME \\\n",
    "  --region=$REGION \\\n",
    "  --module-name=trainer.task \\\n",
    "  --package-path=$PWD/trainer \\\n",
    "  --job-dir=$OUTDIR \\\n",
    "  --staging-bucket=gs://$BUCKET \\\n",
    "  --scale-tier=STANDARD_1 \\\n",
    "  --config=hyperparam.yaml \\\n",
    "  --runtime-version=1.5 \\\n",
    "  -- \\\n",
    "  --train_file_pattern=\"gs://youtube-8m-team/1/video_level/train/train*.tfrecord\" \\\n",
    "  --eval_file_pattern=\"gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\"  \\\n",
    "  --output_dir=$OUTDIR \\\n",
    "  --train_steps=10000 \\\n",
    "  --top_k=5 \\\n",
    "  --job-dir=$OUTDIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Deploy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting and deploying youtube_8m_video_level v1 from gs://youtube8m-4-train/youtube_8m_video_level/trained_model/export/exporter/1527210335/ ... this will take a few minutes\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Created ml engine model [projects/qwiklabs-gcp-8d3d0cd07cef9252/models/youtube_8m_video_level].\n",
      "Creating version (this might take a few minutes)......\n",
      ".......................................................................................................done.\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "MODEL_NAME=\"youtube_8m_video_level\"\n",
    "MODEL_VERSION=\"v1\"\n",
    "MODEL_LOCATION=$(gcloud storage ls gs://$BUCKET/youtube_8m_video_level/trained_model/export/exporter/ | tail -1)\n",    "echo \"Deleting and deploying $MODEL_NAME $MODEL_VERSION from $MODEL_LOCATION ... this will take a few minutes\"\n",
    "#gcloud ml-engine versions delete ${MODEL_VERSION} --model ${MODEL_NAME}\n",
    "#gcloud ml-engine models delete ${MODEL_NAME}\n",
    "gcloud ml-engine models create $MODEL_NAME --regions $REGION\n",
    "gcloud ml-engine versions create $MODEL_VERSION --model $MODEL_NAME --origin $MODEL_LOCATION --runtime-version 1.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "read_dataset_video: _input_fn: file_pattern = gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord\n",
      "read_dataset_video: _input_fn: mode = eval\n",
      "read_dataset_video: _input_fn: batch_size = 1\n",
      "read_dataset_video: _input_fn: features = {'labels': <tensorflow.python.framework.sparse_tensor.SparseTensor object at 0x7f0d50752950>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: features = {'labels': <tf.Tensor 'Cast:0' shape=(4716,) dtype=float32>, 'video_id': <tf.Tensor 'ParseSingleExample/ParseSingleExample:5' shape=() dtype=string>, 'mean_audio': <tf.Tensor 'ParseSingleExample/ParseSingleExample:3' shape=(128,) dtype=float32>, 'mean_rgb': <tf.Tensor 'ParseSingleExample/ParseSingleExample:4' shape=(1024,) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_features = {'video_id': <tf.Tensor 'shuffle_batch:3' shape=(?,) dtype=string>, 'labels': <tf.Tensor 'shuffle_batch:0' shape=(?, 4716) dtype=float32>, 'mean_audio': <tf.Tensor 'shuffle_batch:1' shape=(?, 128) dtype=float32>, 'mean_rgb': <tf.Tensor 'shuffle_batch:2' shape=(?, 1024) dtype=float32>}\n",
      "read_dataset_video: _input_fn: batch_labels = Tensor(\"shuffle_batch:0\", shape=(?, 4716), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Let's call our input function to decode our data to put into BigQuery for testing predictions\n",
    "frame_file_pattern = 'gs://youtube-8m-team/1/video_level/validate/validate-0.tfrecord'\n",
    "\n",
    "fn = read_dataset_video(file_pattern = frame_file_pattern, mode = tf.estimator.ModeKeys.EVAL, batch_size = 1)\n",
    "batch_features, batch_labels = fn()\n",
    "for key,value in batch_features.items():\n",
    "  batch_features[key] = tf.squeeze(batch_features[key])\n",
    "fixed_batch_features = batch_features\n",
    "fixed_batch_labels = tf.squeeze(batch_labels)\n",
    "\n",
    "video_list = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "  sess.run(tf.global_variables_initializer())\n",
    "  sess.run(tf.local_variables_initializer())\n",
    "  coord = tf.train.Coordinator()\n",
    "\n",
    "  threads = tf.train.start_queue_runners(sess = sess, coord = coord)\n",
    "  for _ in range(331):\n",
    "    features, labels = sess.run([fixed_batch_features, fixed_batch_labels])\n",
    "    \n",
    "    features[\"labels\"] = labels\n",
    "    video_list.append(features)\n",
    "\n",
    "  coord.request_stop()\n",
    "  coord.join(threads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "331"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is the number of videos from the video level file we just processed\n",
    "len(video_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Convert the nd-arrays to lists and cast to strings (video_id is already a single string)\n",
    "for items in video_list:\n",
    "  items[\"labels\"] = str(items[\"labels\"].tolist())\n",
    "  items[\"mean_rgb\"] = str(items[\"mean_rgb\"].tolist())\n",
    "  items[\"mean_audio\"] = str(items[\"mean_audio\"].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>mean_rgb</th>\n",
       "      <th>mean_audio</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0MpgkddrY4</td>\n",
       "      <td>[0.3996831178665161, -0.7902982831001282, 1.85...</td>\n",
       "      <td>[1.4524954557418823, 1.5982519388198853, 1.783...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0AjrnQ0N3A</td>\n",
       "      <td>[0.31275367736816406, -0.0275861918926239, -0....</td>\n",
       "      <td>[-1.3102529048919678, -0.16714175045490265, 0....</td>\n",
       "      <td>[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-00LLAtj0JE</td>\n",
       "      <td>[-0.12762920558452606, -0.9175933599472046, -0...</td>\n",
       "      <td>[0.5460372567176819, 1.4538171291351318, -1.59...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-07wapPiIAg</td>\n",
       "      <td>[-1.5168684720993042, 1.3144418001174927, 0.65...</td>\n",
       "      <td>[0.9732653498649597, -0.26707080006599426, -0....</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-02RMEBtLDo</td>\n",
       "      <td>[-0.5323705077171326, -0.18418750166893005, 1....</td>\n",
       "      <td>[-0.3671940267086029, 0.43636152148246765, 1.7...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      video_id                                           mean_rgb  \\\n",
       "0  -0MpgkddrY4  [0.3996831178665161, -0.7902982831001282, 1.85...   \n",
       "1  -0AjrnQ0N3A  [0.31275367736816406, -0.0275861918926239, -0....   \n",
       "2  -00LLAtj0JE  [-0.12762920558452606, -0.9175933599472046, -0...   \n",
       "3  -07wapPiIAg  [-1.5168684720993042, 1.3144418001174927, 0.65...   \n",
       "4  -02RMEBtLDo  [-0.5323705077171326, -0.18418750166893005, 1....   \n",
       "\n",
       "                                          mean_audio  \\\n",
       "0  [1.4524954557418823, 1.5982519388198853, 1.783...   \n",
       "1  [-1.3102529048919678, -0.16714175045490265, 0....   \n",
       "2  [0.5460372567176819, 1.4538171291351318, -1.59...   \n",
       "3  [0.9732653498649597, -0.26707080006599426, -0....   \n",
       "4  [-0.3671940267086029, 0.43636152148246765, 1.7...   \n",
       "\n",
       "                                              labels  \n",
       "0  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...  \n",
       "1  [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...  \n",
       "2  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "3  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "4  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a dataframe from the list\n",
    "video_df = pd.DataFrame(video_list)\n",
    "video_df = video_df[[\"video_id\", \"mean_rgb\", \"mean_audio\", \"labels\"]]\n",
    "video_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Load is 100% Complete\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Export dataframe to BigQuery\n",
    "import datalab.bigquery as bq\n",
    "bigquery_dataset_name = 'ryan_youtube'\n",
    "bigquery_table_name = 'tbl_video_level'\n",
    "\n",
    "# Define BigQuery dataset and table\\n\",\n",
    "dataset = bq.Dataset(bigquery_dataset_name)\n",
    "table = bq.Table(bigquery_dataset_name + '.' + bigquery_table_name)\n",
    "\n",
    "# Create BigQuery dataset\n",
    "if not dataset.exists():\n",
    "    dataset.create()\n",
    "\n",
    "# Create or overwrite the existing table if it exists\\n\",\n",
    "table_schema = bq.Schema.from_dataframe(video_df)\n",
    "table.create(schema = table_schema, overwrite = True)\n",
    "\n",
    "video_df.to_gbq(destination_table = bigquery_dataset_name + '.' + bigquery_table_name, project_id = \"qwiklabs-gcp-8d3d0cd07cef9252\", if_exists = \"replace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Create SQL query\n",
    "query=\"\"\"\n",
    "SELECT\n",
    "  video_id,\n",
    "  mean_rgb,\n",
    "  mean_audio\n",
    "FROM\n",
    "  `qwiklabs-gcp-8d3d0cd07cef9252.ryan_youtube.tbl_video_level`\n",
    "LIMIT\n",
    "  3\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>mean_rgb</th>\n",
       "      <th>mean_audio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0FDy3F9Gqo</td>\n",
       "      <td>[0.5610039830207825, 0.6452203989028931, 0.365...</td>\n",
       "      <td>[-1.1589237451553345, 0.8263356685638428, 0.21...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0CK7kXtP-U</td>\n",
       "      <td>[0.18520089983940125, -0.03229318931698799, -0...</td>\n",
       "      <td>[0.4032512307167053, -0.6461716294288635, 0.08...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0QTVHnvT90</td>\n",
       "      <td>[0.5585809350013733, 0.09270916134119034, -0.2...</td>\n",
       "      <td>[-0.4419490396976471, -0.3150809705257416, 0.7...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      video_id                                           mean_rgb  \\\n",
       "0  -0FDy3F9Gqo  [0.5610039830207825, 0.6452203989028931, 0.365...   \n",
       "1  -0CK7kXtP-U  [0.18520089983940125, -0.03229318931698799, -0...   \n",
       "2  -0QTVHnvT90  [0.5585809350013733, 0.09270916134119034, -0.2...   \n",
       "\n",
       "                                          mean_audio  \n",
       "0  [-1.1589237451553345, 0.8263356685638428, 0.21...  \n",
       "1  [0.4032512307167053, -0.6461716294288635, 0.08...  \n",
       "2  [-0.4419490396976471, -0.3150809705257416, 0.7...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Export BigQuery results to dataframe\n",
    "import google.datalab.bigquery as bq2\n",
    "df_predict = bq2.Query(query).execute().result().to_dataframe()\n",
    "df_predict.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Local prediction from local model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Format dataframe to new line delimited json strings and write out to json file\n",
    "with open('video_level.json', 'w') as outfile:\n",
    "  for idx, row in df_predict.iterrows():\n",
    "    json_string = \"{\\\"video_id\\\": \\\"\" + row[\"video_id\"] + \"\\\", \\\"mean_rgb\\\": \\\"\" + row[\"mean_rgb\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\") + \"\\\", \\\"mean_audio\\\": \\\"\" + row[\"mean_audio\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\") + \"\\\"}\"\n",
    "    outfile.write(\"%s\\n\" % json_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLASSES           LOGITS                                                                                                     PREDICTIONS                PROBABILITIES\n",
      "[7, 3, 6, 1, 8]   [-0.5442675352096558, -0.7138631343841553, -0.8265247344970703, -1.0281424522399902, -1.1685709953308105]  [1.0, 1.0, 1.0, 1.0, 1.0]  [0.36719539761543274, 0.3287458121776581, 0.3043803870677948, 0.2634443938732147, 0.2371133714914322]\n",
      "[1, 13, 3, 7, 6]  [-1.2642022371292114, -2.220372438430786, -2.6308624744415283, -2.9802935123443604, -3.3348684310913086]   [1.0, 1.0, 1.0, 1.0, 1.0]  [0.22025136649608612, 0.09793590009212494, 0.0671783834695816, 0.04832413047552109, 0.03439417481422424]\n",
      "[1, 13, 3, 7, 6]  [-0.5264356136322021, -0.95451420545578, -1.23331618309021, -1.344706654548645, -1.3451508283615112]       [1.0, 1.0, 1.0, 1.0, 1.0]  [0.37134861946105957, 0.2779778838157654, 0.22560153901576996, 0.20673711597919464, 0.2066642940044403]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: /usr/local/envs/py2env/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2018-05-25 02:45:51.882707: I tensorflow/core/platform/cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%bash\n",
    "model_dir=$(ls $PWD/trained_model/export/exporter | tail -1)\n",
    "gcloud ml-engine local predict \\\n",
    "    --model-dir=$PWD/trained_model/export/exporter/$model_dir \\\n",
    "    --json-instances=./video_level.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### GCloud ML-Engine prediction from deployed model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Format dataframe to instances list to get sent to ML-Engine\n",
    "instances = [{\"video_id\": row[\"video_id\"], \"mean_rgb\": row[\"mean_rgb\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\"), \"mean_audio\": row[\"mean_audio\"].replace(\" \",\"\").replace(\"[\",\"\").replace(\"]\",\"\")} for idx, row in df_predict.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "response = {u'predictions': [{u'probabilities': [0.11992227286100388, 0.1173163652420044, 0.0829235389828682, 0.07739514857530594, 0.06102382391691208], u'logits': [-1.993166446685791, -2.018092632293701, -2.4032719135284424, -2.4782769680023193, -2.7335257530212402], u'classes': [0, 1, 3, 2, 5], u'predictions': [1.0, 1.0, 1.0, 1.0, 1.0]}, {u'probabilities': [0.11992227286100388, 0.1173163652420044, 0.0829235389828682, 0.07739514857530594, 0.06102382391691208], u'logits': [-1.993166446685791, -2.018092632293701, -2.4032719135284424, -2.4782769680023193, -2.7335257530212402], u'classes': [0, 1, 3, 2, 5], u'predictions': [1.0, 1.0, 1.0, 1.0, 1.0]}, {u'probabilities': [0.11992227286100388, 0.1173163652420044, 0.0829235389828682, 0.07739514857530594, 0.06102382391691208], u'logits': [-1.993166446685791, -2.018092632293701, -2.4032719135284424, -2.4782769680023193, -2.7335257530212402], u'classes': [0, 1, 3, 2, 5], u'predictions': [1.0, 1.0, 1.0, 1.0, 1.0]}]}\n"
     ]
    }
   ],
   "source": [
    "# Send instance dictionary to receive response from ML-Engine for online prediction\n",
    "from googleapiclient import discovery\n",
    "from oauth2client.client import GoogleCredentials\n",
    "import json\n",
    "\n",
    "credentials = GoogleCredentials.get_application_default()\n",
    "api = discovery.build('ml', 'v1', credentials=credentials)\n",
    "\n",
    "request_data = {\"instances\": instances}\n",
    "\n",
    "parent = 'projects/%s/models/%s/versions/%s' % (PROJECT, 'youtube_8m_video_level', 'v1')\n",
    "response = api.projects().predict(body = request_data, name = parent).execute()\n",
    "print(\"response = {}\".format(response))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
